package y1s.servlet;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.*;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import y1s.controller.Utils;
import y1s.data.Data;

import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;



public class GetPage extends Thread{
	DBObject domain;
	public GetPage(DBObject domain)
	{
		this.domain=domain;;
		this.start();
	}
	public void run()
	{
		try {

			URL dataURL = new URL("http://"+domain.get("name"));			
			String charset   = "UTF-8";		
			String content = "";
			BufferedReader reader;
			
			try {
				content = ReturnContent(domain.get("name").toString());
			} catch (Exception e) {
				// TODO: handle exception
			}
			
			if(content == ""){
				content = ReturnContent("www."+domain.get("name").toString());
				dataURL = new URL("http://www."+domain.get("name"));
			}

			Document doc = Jsoup.parse(content);
			
			int size = doc.html().length();
			
			Elements link = doc.select("a");
			
			Long time1 = new Date().getTime();
			
			final int index = doc.select("meta").toString().toLowerCase( ).indexOf( "charset=" );
			if(index != -1)
				charset = doc.select("meta").toString().substring( index+8 ).replaceAll("[^\\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "").replaceAll("[\\\'\\\"]+", "");
			
			HttpURLConnection connections = (HttpURLConnection) dataURL.openConnection();
			reader = new BufferedReader(new InputStreamReader(connections.getInputStream(), Charset.forName(charset)));

			String server = connections.getHeaderField("Server");
			String powerBy = connections.getHeaderField("X-Powered-By");
			
			domain.put("server", server);
			domain.put("language", powerBy);
			
			content = "";			
			while(1==1)
			{
				String str = reader.readLine();
				if(str==null) break;
				content+=str;				
			}
			Long time2 = new Date().getTime();
			doc = Jsoup.parse(content);			
			String title = "";
			String desc=null;
			String key = null;
			int mdesc = 0;
			int mkey = 0;
			try
			{				
				try {
					title = doc.select("title").first().text();
				} catch (Exception e) {
					// TODO: handle exception
				}

				try {
					key = doc.select("meta[name=keywords]").first().attr("content");
					mkey = 1;
				} catch (Exception e) {
					try {
						key = doc.select("meta[http-equiv=keywords]").first().attr("content");
					} catch (Exception e2) {
						// TODO: handle exception
					}					
				}				
				try {
					desc = doc.select("meta[name=description]").first().attr("content");
					mdesc = 1;
				} catch (Exception e) {
					try {
						desc = doc.select("meta[http-equiv=description]").first().attr("content");
					} catch (Exception e2) {
						try {
							if(doc.select("body").first().text().length() > 260)
							{
								desc = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "");
							} else {
								desc = doc.select("body").first().text();
							}
						} catch (Exception e3) {
							// TODO: handle exception
						}
					}
				}
				if(desc!=null&&desc.length()>260)
				{
					desc = desc.substring(0,250);
				}
			}
			catch(Exception ex)
			{
				ex.printStackTrace();
				
			}
			
			int begin = content.indexOf("_gaq.push(['_setAccount', '");
			String ananyticAccount = "";
			if(begin>0)
			{
				begin+="_gaq.push(['_setAccount', '".length();
				int end =  content.indexOf("'",begin);
				ananyticAccount = content.substring(begin, end);
				begin=ananyticAccount.lastIndexOf("-")-1;
				
				ananyticAccount = ananyticAccount.substring(0, begin);			
			}
			else
			{				
				begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
				if(begin>0)
				{
					begin+="_gaq.push([\"_setAccount\", \"".length();
					int end =  content.indexOf("\"",begin);
					ananyticAccount = content.substring(begin, end);
				} else {
					begin = content.indexOf("_uacct");
					if(begin > 0)
					{
						begin+="_uacct".length();
						ananyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
						begin=ananyticAccount.lastIndexOf("-");						
						ananyticAccount = ananyticAccount.substring(0, begin);
					} else {
						Pattern p = Pattern.compile("UA-[0-9]+");
				        Matcher m = p.matcher(content);
				        while (m.find()) {
				        	ananyticAccount = m.group();
				        }
					}
				}
			}
			begin = content.indexOf("google_ad_client = \"");
			String adsAccount = "";
			if(begin>0)
			{
				begin+="google_ad_client = \"', '".length();
				int end =  content.indexOf("\"",begin);
				adsAccount = content.substring(begin, end);
			}

			domain.put("title", title);
			if(desc != null)
				domain.put("description", desc);
			domain.put("domain", Utils.returnDomain(domain.get("name").toString()));
			domain.put("googleAnalytics", ananyticAccount);
			domain.put("googleAdsense", adsAccount);
			
			BasicDBObject b_key = new BasicDBObject();
			
			ArrayList<String> l_keywords = new ArrayList<String> ();

				
			if(key!=null)
			{
				String[] keywords = key.split(",");
				for(int i=0;i<keywords.length;i++)
				{
					if(!l_keywords.contains(keywords[i].trim()))
					{
						l_keywords.add(keywords[i].trim());
						b_key.put(keywords[i].trim().replaceAll("[.]+", "&&&"), 1);
					}
					
				}
			}
			
			domain.put("lKeyword", l_keywords);
			
			BasicDBObject b_email = new BasicDBObject();
			
			try
			{
				// email				
				final String RE_MAIL = "([\\w\\-]([\\.\\w])+[\\w]+@([\\w\\-]+\\.)+[A-Za-z]{2,4})";
			    Pattern p = Pattern.compile(RE_MAIL);
			    Matcher m = p.matcher(content.replaceAll("\\<.*?\\>", " "));

			    while(m.find()) {
			    	if(m.group(1).endsWith(domain.get("name").toString()))
			    	{	
			    		b_email.put(m.group(1).toLowerCase().replaceAll("[.]+", "&&&"), 1);
			    	}				    	
			    } 
			}
			catch (Exception ex)
			{
				 ex.printStackTrace();
				 
			}
			domain.put("email", b_email);
			
			// JS
			BasicDBObject l_js = new BasicDBObject();
			if(content.indexOf("jquery")>0)
				l_js.put("jquery",1);
			if(content.indexOf("prototype.js")>0)
				l_js.put("prototype",1);
			if(content.indexOf("extjs")>0)
				l_js.put("extjs",1);
			if(content.indexOf("yui")>0)
				l_js.put("yui",1);
			if(content.indexOf("dojo")>0)
				l_js.put("dojo",1);
			if(content.indexOf("gwt")>0)
				l_js.put("gwt",1);
			if(content.indexOf("easeljs")>0)
				l_js.put("easeljs",1);
			
			domain.put("js", l_js);
			//phan tich the
			Elements element = doc.select("h1");
			domain.put("h1", element.size());
			element = doc.select("h2");	
			domain.put("h2", element.size());
			element = doc.select("h3");	
			domain.put("h3", element.size());
			element = doc.select("h4");	
			domain.put("h4", element.size());
			element = doc.select("h5");	
			domain.put("h5", element.size());
			element = doc.select("img");
			
			int count = 0;
			for(int i=0;i<element.size();i++)
			{
				if(element.get(i).attr("alt") != null && !element.get(i).attr("alt").equals(""))
				{
					count++;
				}
			}
			domain.put("altImg", count);
			domain.put("img", element.size());
			element = doc.select("embed");
			domain.put("flash", element.size());
			element = doc.select("frame");
			domain.put("frame", element.size());
			domain.put("charset", charset);				
			
			domain.put("metaDesc", mdesc);
			domain.put("metaKey", mkey);
			domain.put("size", size);
			domain.put("speed",((float)(time2-time1))/1000);

			ArrayList<String> listLink = new ArrayList<String>();
			String check = null;

			BasicDBObject link_ob = new BasicDBObject();
			
			for(int i=0;i<link.size();i++)
			{
				check = link.get(i).attr("rel"); 
				if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
				{
					if( check != null && !check.equals("nofollow") || check == null)
					{
						Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
	
				        Matcher m = p.matcher(link.get(i).attr("href"));
				        DBCollection coll = Data.data("link");
				        while (m.find()) {
				        	String dom = m.group().replaceAll("(http|https)+://(www.)*", "").replaceAll("[\\?]+[\\w\\W]+", "").replaceAll("[\\/]+[\\w\\W]+", "");
				        	if(!listLink.contains(dom) && !dom.equals(domain.get("name")))
				        	{
				        		listLink.add(dom);
				        		link_ob.put(dom.replaceAll("[.]+", "&&&").toLowerCase().trim(), 1);
				        		
				        		DBObject dbCheck = coll.findOne(new BasicDBObject("name",domain.get("name")).append("link", dom.toLowerCase().trim()));
				        		if(dbCheck == null)
				        		{
					        		BasicDBObject inLink = new BasicDBObject();
					        		inLink.put("name", domain.get("name"));
					        		inLink.put("link", dom.toLowerCase().trim());
					        		coll.insert(inLink);
				        		}
				        	}
				        }
					}
				}
			}

			domain.put("link", link_ob);

		}
		catch (Exception e) {
			
			e.printStackTrace();
		} 
	}
	public static String ReturnContent(String domain) throws IOException
	{
		String content = "";
		String charset   = "UTF-8";	
		URL dataURL = new URL("http://"+domain);
		HttpURLConnection connection = (HttpURLConnection) dataURL.openConnection();
	    connection.setInstanceFollowRedirects(true);
	    connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.21) Gecko/20110830 AlexaToolbar/alxf-2.14 AlexaToolbar/pXVUnvuf-1.2 Firefox/3.6.21");
		connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
		connection.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
		connection.setRequestProperty("Accept-Charset", "	ISO-8859-1,utf-8;q=0.7,*;q=0.7");
		connection.setRequestProperty("Connection", "keep-alive");
		connection.setRequestProperty("Keep-Alive", "115");
		connection.setRequestProperty("Cookie", "PREF=ID=987bb7a56b1cc7bc:U=691fa246d56f502e:FF=0:TM=1314876638:LM=1314876681:S=yBXzqp_WQq9ykX-j; NID=50=nfNXg7ymSSsfdzMnkEuo7t1GCgIEwcK5xmpFhu1mQWwvNI5nFHLZpRsy_1XnaoKrdj9hnz7za9-M2OIO99phfFSTmXln-xCvqzJaRWHsN0EQKgpmkvgUDTWPVF4I4c9L; SID=DQAAALcAAADhqr-J8Ls3CKhmbRjwTMMP1DjmIvR8qBFxv99saTaFFgoRsVf18W4p_BTocYoxYNMvsvh9sfjMj7KqLyd_s9Y0lwYY_Rrljd7t5IpKBTWWnsv9VSJ_qelH943zV8YIT3uuIdJbHhFlCnVEu_YealZoldWjf4VirYR0PIrr0i4_JU2JCHRmpyW0ACt3KcyLkukJlGDyT5yC5js3CBSTAhnWSylBpMWPWyWJzqXe2BQFcdfuU_qo00pAR-uEcMQ437M; HSID=A9Z-j3CKhzDEykAES");
		connection.setRequestProperty("Cache-Control", "max-age=0");
	    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName(charset)));
		
		char[] buf = new char[102102];
		int numRead=0;
        while((numRead=reader.read(buf)) != -1){
            String readData = String.valueOf(buf, 0, numRead);
            content += readData;
            buf = new char[102102];
        }
        reader.close();
        
		return content;
	}
//	public static void main( String args[])
//	{
//		new SelfSiteUpdate("mintworks.com");
//	}
}
